﻿using HtmlAgilityPack;
using Newtonsoft.Json;
using NLog.Fluent;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace AreaCodeConsoleApp
{
    class Program
    {
        static ReaderWriterLockSlim LogWriteLock = new ReaderWriterLockSlim();
        static void Main(string[] args)
        {
            Log.logger.Info("start...");
            Log.logger.Error("start...");
            Console.WriteLine("爬取国家行政区划数据 主页地址 " + Globalvariable.UrlIndex);
            Console.WriteLine("敲回车开始爬取...");
            Console.ReadKey();

            Bussine bussine = new Bussine();
            bussine.LogWriteLock = LogWriteLock;
            List<AreaInfo> list_Province = bussine.GetProvince();
            bussine.WriteFile(list_Province);
            bussine.PrintConsole(list_Province);

            foreach (var itemProvince in list_Province)
            {
                List<AreaInfo> list_City = bussine.GetCity(itemProvince.UrlNext, itemProvince);
                bussine.WriteFile(list_City);
                bussine.PrintConsole(list_City);
                foreach (var itemCity in list_City)
                {
                    bool IsNoArea = bussine.NoAreaForCityName().Where(table => table.IndexOf(itemCity.Name) != -1).Any();
                    if (IsNoArea)
                    {
                        Console.WriteLine("城市 " + itemCity.Name + "无地区，从街道取数据。");
                        Log.logger.Info("城市 " + itemCity.Name + "无地区，从街道取数据。");
                        bussine.NoArea(itemCity);
                        continue;
                    }

                    List<AreaInfo> list_Area = bussine.GetArea(itemCity.UrlNext, itemCity);
                    bussine.WriteFile(list_Area);
                    bussine.PrintConsole(list_Area);

                    foreach (var itemarea in list_Area)
                    {
                        if (!string.IsNullOrEmpty(itemarea.UrlNext)) //区下没有街道和居委的情况下
                        {
                            List<AreaInfo> list_Street = bussine.GetStreet(itemarea.UrlNext, itemarea);
                            bussine.WriteFile(list_Street);
                            bussine.PrintConsole(list_Street);

                            foreach (var itemStreet in list_Street)
                            {
                                List<AreaInfo> list_Community = bussine.GetCommunity(itemStreet.UrlNext, itemStreet);
                                bussine.WriteFile(list_Community);
                                bussine.PrintConsole(list_Community);
                                Thread.Sleep(500);
                            }

                        }
                        else
                        {
                            StringBuilder sb = new StringBuilder();
                            sb.AppendFormat("区:{0}_{1} 等级{2} 没有下级数据", itemarea.Name, itemarea.Code, itemarea.Level);
                            Log.logger.Info(sb.ToString());
                            Console.WriteLine(sb.ToString());
                        }
                        Thread.Sleep(500);
                    }
                    Thread.Sleep(500);
                }
                Console.WriteLine(itemProvince.Name + "_" + itemProvince.Code + "数据抓取完毕!");
                Log.logger.Info(itemProvince.Name + "_" + itemProvince.Code + "数据抓取完毕!");
            }
            Console.WriteLine("等待后台线程，执行完成毕。。");
            Console.ReadKey();
        }

        static void Exct(object model)
        {

        }
    }

    public class Globalvariable
    {
        /// <summary>
        /// 国家行政区划最新地址
        /// </summary>
        public const string UrlIndex = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/index.html";
    }

    public class Bussine
    {
        public ReaderWriterLockSlim LogWriteLock { get; set; }
        public void WriteFile(List<AreaInfo> list)
        {
            try
            {
                LogWriteLock.EnterWriteLock();
                string filename = Path.Combine(System.Environment.CurrentDirectory + "\\Area.json");
                StreamWriter fs = new StreamWriter(filename, true);
                string json = "";
                for (int i = 0; i < list.Count; i++)
                {
                    json += JsonConvert.SerializeObject(list[i]) + ",\n";
                }
                fs.Write(json);
                fs.Close();
            }
            catch (Exception ex)
            {
                Log.logger.Error("写入文件异常：" + ex.Message);
            }
            finally
            {
                LogWriteLock.ExitWriteLock();
            }
        }
        public void PrintConsole(List<AreaInfo> list)
        {
            string json = "";
            for (int i = 0; i < list.Count; i++)
            {
                json += JsonConvert.SerializeObject(list[i]) + ",\n";
            }
            Console.WriteLine(json);
        }
        public List<AreaInfo> GetProvince()
        {
            string search = "//tr[@class = 'provincetr']/td/a";
            List<AreaInfo> result = null;
            HtmlNodeCollection colletion = GetCollection(Globalvariable.UrlIndex, search);
            while (colletion == null)
            {
                Thread.Sleep(1000 * 10);//休眠10秒
                colletion = GetCollection(Globalvariable.UrlIndex, search);
            }

            if (null != colletion)
                result = GetList(1, "", Globalvariable.UrlIndex, colletion);

            return result;
        }
        public List<AreaInfo> GetCity(string url, AreaInfo model)
        {
            string search = "//tr[@class = 'citytr']";
            List<AreaInfo> result = null;
            HtmlNodeCollection colletion = GetCollection(model.UrlNext, search);
            while (colletion == null)
            {
                StringBuilder sb = new StringBuilder();
                sb.AppendFormat("网络异常 {0} 等级{1},地址:{2}", model.Code + "_" + model.Name, model.Level, model.UrlNext);
                Console.WriteLine(sb);
                Log.logger.Info(sb);
                Thread.Sleep(1000 * 10);//休眠10秒
                colletion = GetCollection(model.UrlNext, search);
            }

            if (null != colletion)
                result = GetList(2, model.Code, url, colletion);

            return result;
        }
        public List<AreaInfo> GetArea(string url, AreaInfo model)
        {
            string search = "//tr[@class = 'countytr']";
            List<AreaInfo> result = null;
            HtmlNodeCollection colletion = GetCollection(model.UrlNext, search);
            while (colletion == null)
            {
                StringBuilder sb = new StringBuilder();
                sb.AppendFormat("网络异常 {0} 等级{1},地址:{2}", model.Code + "_" + model.Name, model.Level, model.UrlNext);
                Console.WriteLine(sb);
                Log.logger.Info(sb);
                Thread.Sleep(1000 * 10);//休眠10秒
                colletion = GetCollection(model.UrlNext, search);
            }

            if (null != colletion)
                result = GetList(3, model.Code, url, colletion);

            return result;
        }
        public List<AreaInfo> GetStreet(string url, AreaInfo model)
        {
            string search = "//tr[@class = 'towntr']";
            List<AreaInfo> result = null;
            HtmlNodeCollection colletion = GetCollection(model.UrlNext, search);
            while (colletion == null)
            {
                StringBuilder sb = new StringBuilder();
                sb.AppendFormat("网络异常 {0} 等级{1},地址:{2}", model.Code + "_" + model.Name, model.Level, model.UrlNext);
                Console.WriteLine(sb);
                Log.logger.Info(sb);
                Thread.Sleep(1000 * 10);//休眠10秒
                colletion = GetCollection(model.UrlNext, search);
            }

            if (null != colletion)
                result = GetList(4, model.Code, url, colletion);

            return result;
        }
        public List<AreaInfo> GetCommunity(string url, AreaInfo model)
        {
            string search = "//tr[@class = 'villagetr']";
            List<AreaInfo> result = null;
            HtmlNodeCollection colletion = GetCollection(model.UrlNext, search);
            while (colletion == null)
            {
                StringBuilder sb = new StringBuilder();
                sb.AppendFormat("网络异常 {0} 等级{1},地址:{2}", model.Code + "_" + model.Name, model.Level, model.UrlNext);
                Console.WriteLine(sb);
                Log.logger.Info(sb);
                Thread.Sleep(1000 * 10);//休眠10秒
                colletion = GetCollection(model.UrlNext, search);
            }

            if (null != colletion)
                result = GetList(5, model.Code, url, colletion);

            return result;
        }
        public List<AreaInfo> GetList(int level, string ParentCode, string url, HtmlNodeCollection collection)
        {
            List<AreaInfo> list = new List<AreaInfo>();
            foreach (var item in collection)
            {
                AreaInfo info = new AreaInfo();
                List<HtmlNode> HtmlNodelist = item.Elements("td").ToList();
                switch (level)
                {
                    case 1:

                        info.Name = item.InnerText;
                        string urlNext = item.Attributes["href"].Value;
                        info.Code = urlNext.Substring(0, urlNext.IndexOf(".")) + "0000000000";
                        info.ParentCode = "";
                        info.UrlNext = url.Substring(0, url.LastIndexOf("/") + 1) + urlNext;
                        info.Level = (int)Arealevel.Province;
                        list.Add(info);
                        break;
                    case 2:
                        if (null != HtmlNodelist && HtmlNodelist.Count == 2)
                        {
                            info.Code = HtmlNodelist[0].InnerText;
                            info.Name = HtmlNodelist[1].InnerText;
                            string temurl = HtmlNodelist[0].Elements("a").ToList()[0].Attributes["href"].Value;
                            info.ParentCode = ParentCode;
                            info.Level = (int)Arealevel.City;
                            info.UrlNext = url.Substring(0, url.LastIndexOf("/") + 1) + temurl;
                            list.Add(info);
                        }
                        break;
                    case 3:
                        if (null != HtmlNodelist && HtmlNodelist.Count == 2)
                        {
                            info.Code = HtmlNodelist[0].InnerText;
                            info.Name = HtmlNodelist[1].InnerText;
                            info.ParentCode = ParentCode;
                            info.Level = (int)Arealevel.Area;
                            if (HtmlNodelist[0].Elements("a").ToList().Count > 0)
                            {
                                string temurl = HtmlNodelist[0].Elements("a").ToList()[0].Attributes["href"].Value;
                                info.UrlNext = url.Substring(0, url.LastIndexOf("/") + 1) + temurl;
                            }
                            list.Add(info);
                        }
                        break;
                    case 4:
                        if (null != HtmlNodelist && HtmlNodelist.Count == 2)
                        {
                            string code = HtmlNodelist[0].InnerText;
                            string name = "";
                            if (HtmlNodelist[1].InnerText.IndexOf("办事处") != -1)
                                name = HtmlNodelist[1].InnerText.Substring(0, HtmlNodelist[1].InnerText.IndexOf("办事处"));
                            else
                                name = HtmlNodelist[1].InnerText;

                            string temurl = HtmlNodelist[0].Elements("a").ToList()[0].Attributes["href"].Value;
                            info.ParentCode = ParentCode;
                            info.Level = (int)Arealevel.Street;
                            info.Code = code;
                            info.Name = name;
                            info.UrlNext = url.Substring(0, url.LastIndexOf("/") + 1) + temurl;
                            list.Add(info);
                        }
                        break;
                    case 5:
                        if (null != HtmlNodelist && HtmlNodelist.Count == 3)
                        {
                            string code = HtmlNodelist[0].InnerText;
                            string name = HtmlNodelist[2].InnerText;
                            AreaInfo tem = new AreaInfo();
                            tem.ParentCode = ParentCode;
                            tem.Level = (int)Arealevel.Community;
                            tem.Code = code;
                            tem.Name = name;
                            list.Add(tem);
                        }
                        break;
                    default:
                        break;
                }
            }
            return list;
        }
        public HtmlNodeCollection GetCollection(string url, string search)
        {
            HtmlNodeCollection colletion = null;
            HttpWebRequest request = null;
            try
            {
                request = (HttpWebRequest)HttpWebRequest.Create(url);
                request.KeepAlive = false;
                request.Proxy = null;
                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
                request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3";
                request.ContentType = "application/x-www-form-urlencoded";
                request.CookieContainer = new CookieContainer();
                request.ContinueTimeout = 2000;
                request.Timeout = 2000;
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                using (StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.Default))
                {
                    string html = reader.ReadToEnd();
                    HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
                    document.LoadHtml(html);
                    colletion = document.DocumentNode.SelectNodes(search);
                }
                response.Close();
                response.Dispose();
            }
            catch (Exception ex)
            {
                Console.WriteLine("地址：" + url + "网络异常" + ex.Message);
                Log.logger.Error("地址：" + url + "网络异常" + ex.Message);
            }
            finally
            {
                if (null != request)
                {
                    request.Abort();
                }
            }
            return colletion;
        }

        public void NoArea(AreaInfo CityItem)
        {
            List<AreaInfo> list_Street = GetStreet(CityItem.UrlNext, CityItem);
            WriteFile(list_Street);
            PrintConsole(list_Street);
            foreach (var itemStreet in list_Street)
            {
                Thread.Sleep(1000);
                List<AreaInfo> list_Community = GetCommunity(itemStreet.UrlNext, itemStreet);
                WriteFile(list_Community);
                PrintConsole(list_Community);
            }
        }

        public List<string> NoAreaForCityName()
        {
            return new List<string>() { "东莞市", "中山市", "儋州市" };
        }

    }
    public enum Arealevel
    {
        /// <summary>
        /// 省 一级
        /// </summary>
        Province = 1,
        /// <summary>
        /// 市 二级
        /// </summary>
        City = 2,
        /// <summary>
        /// 区/县 三级
        /// </summary>
        Area = 3,
        /// <summary>
        /// 街道/镇 四级
        /// </summary>
        Street = 4,
        /// <summary>
        /// 居委/村委 五级
        /// </summary>
        Community = 5
    }
    public class AreaInfo
    {
        public string Name { get; set; }
        public string Code { get; set; }
        public int Level { get; set; }
        public string ParentCode { get; set; }
        public string Remark { get; set; }
        [JsonIgnore]
        public string UrlNext { get; set; }
    }
}
